﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Net;
using HtmlParserLib;
using System.Diagnostics;

namespace NDownloader
{
    public static class URLScan
    {
        public static List<String> GetLinks(String strURL, String[] strExtensions)
        {
            List<String> lstStrURLs = new List<String>();
            try
            {

                // Download page
                WebClient client = new WebClient();
                string strHtml = client.DownloadString(strURL);

                // Scan links on this page
                HtmlTag tag;
                HtmlParserLib.HtmlParser parse = new HtmlParserLib.HtmlParser(strHtml);
                while (parse.ParseNext("a", out tag))
                {
                    // See if this anchor links to us
                    string value;
                    string extension;
                    if (tag.Attributes.TryGetValue("href", out value))
                    {
                        // value contains URL referenced by this link
                        //select the links with the given extensions
                        //Remove the . from the extention since the config file does not have them
                        extension = System.IO.Path.GetExtension(value).ToString();
                        if (extension.Length > 0)
                        {
                            extension = extension.Substring(1);
                            if (strExtensions.Contains(extension))
                            {
                                lstStrURLs.Add(value);
                            }                                               
                        }
                    }
                }                               
            }
            catch( Exception ex)
            {
                Debug.WriteLine(ex.ToString());
                Console.WriteLine(ex.ToString()); 
            }

            return lstStrURLs;
        }
    }
}